{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "登陆成功，若报错，请用文件夹进入一次\\10.153.110.37\n",
      "正在进行定时数据分析程序，若要修改定时的时间，请看最后注释\n",
      "2021-10-06 18:30:00\n",
      "None\n",
      "开始执行定时数据分析\n",
      "正在处理cak文件\n",
      "目标地址：\\\\10.153.110.37\\Data\\solar_observation\\2021\\CAK\\20211006CAK\n",
      "已完成，文件放置日期处理目录：\\\\10.153.110.37\\Data\\solar_observation\\2021\\CAK\\20211006CAK\n",
      "数据已保存至：\\\\10.153.110.37\\Data\\solar_observation\\2021\\CAK\\\n",
      "**************************************************\n",
      "正在处理wl文件\n",
      "目标地址：\\\\10.153.110.37\\Data\\solar_observation\\2021\\WL\\20211006WL\n",
      "已完成，文件放置日期处理目录：\\\\10.153.110.37\\Data\\solar_observation\\2021\\WL\\20211006WL\n",
      "数据已保存至：\\\\10.153.110.37\\Data\\solar_observation\\2021\\WL\n",
      "**************************************************\n",
      "正在处理ha文件\n",
      "目标地址：\\\\10.153.110.37\\Data\\solar_observation\\2021\\HA\\20211006HA\n",
      "已完成，文件放置日期处理目录：\\\\10.153.110.37\\Data\\solar_observation\\2021\\HA\\20211006HA\n",
      "数据已保存至：\\\\10.153.110.37\\Data\\solar_observation\\2021\\HA\n",
      "**************************************************\n",
      "数据分析完成，期待明天有好天气\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import os,datetime,shutil,schedule,time\n",
    "import datetime as dt\n",
    "import pandas as pd\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "\n",
    "print('登陆成功，若报错，请用文件夹进入一次\\\\10.153.110.37')\n",
    "print(f'正在进行定时数据分析程序，若要修改定时的时间，请看最后注释')\n",
    "os.chdir(r'\\\\10.153.110.37\\Data\\solar_observation\\2021')\n",
    "\n",
    "# def job():\n",
    "#     print(print(time.strftime(\"%Y-%m-%d %H:%M:%S\", time.localtime())))\n",
    "#     print('开始执行定时备份')\n",
    "\n",
    "网盘37字典={}\n",
    "def job():\n",
    "    print(print(time.strftime(\"%Y-%m-%d %H:%M:%S\", time.localtime())))\n",
    "    print('开始执行定时数据分析')\n",
    "\n",
    "        #如果观测时间分上下午，上午时间一个列表，下午时间一个列表，分开求和再相加得出总的时间\n",
    "    def 文件夹日期(path):\n",
    "        return re.search('\\d+',path).group()\n",
    "\n",
    "    def 时间计算(time):\n",
    "        if time != []:\n",
    "            a = time[-1]\n",
    "            b = time[0]\n",
    "            c = round((a-b).seconds/60/60,2)\n",
    "            return round(c,2)\n",
    "        else:\n",
    "            return 0\n",
    "    def 时间段(time):\n",
    "        if time != []:\n",
    "            b=time[0]\n",
    "            c=time[-1]\n",
    "            return '-'.join((b.strftime('%X'),c.strftime('%X')))\n",
    "        else:\n",
    "            return None\n",
    "\n",
    "\n",
    "    #爬取太阳黑子，网站https://www.solarmonitor.org/index.php?date=20211023\n",
    "    def 黑子(num):\n",
    "        res = requests.get('https://www.solarmonitor.org/index.php?date='+str(num))\n",
    "        bs = BeautifulSoup(res.text,'html.parser')\n",
    "        noaa=bs.find_all('tr',class_='noaaresults')\n",
    "        if noaa ==[]:\n",
    "            #编号\n",
    "            编号.append('平静')\n",
    "            #方位\n",
    "            方位.append('平静')\n",
    "            #黑子数量\n",
    "            黑子数量.append('无黑子')\n",
    "            #positions = position.replace('S','+').replace('N','-')   #需求要把南北换成 +\\-\n",
    "            #耀斑爆发等级和时间\n",
    "            耀斑爆发等级和时间.append('平静')\n",
    "        else:\n",
    "            a=[]\n",
    "            b=[]\n",
    "            c=[]\n",
    "            d=[]\n",
    "            for s in noaa:\n",
    "                #编号\n",
    "                a.append(s.find(id='noaa_number').text.replace(' ',''))\n",
    "                #方位\n",
    "                b.append(s.find(id='position').text.replace(' ',''))\n",
    "                #黑子数量\n",
    "                c.append(s.find(id='nspots').text.replace(' ',''))\n",
    "                #positions = position.replace('S','+').replace('N','-')   #需求要把南北换成 +\\-\n",
    "                #耀斑爆发等级和时间\n",
    "                d.append(s.find(id='events').text.replace(' ',''))\n",
    "            编号.append(a)\n",
    "            方位.append(b)\n",
    "            黑子数量.append(c)\n",
    "            耀斑爆发等级和时间.append(d)\n",
    "\n",
    "\n",
    "    def 合并():\n",
    "        file37 = ppath1+'\\文件整理表格.xls'\n",
    "        df37 = pd.read_excel(file37)\n",
    "        fileday = ppath+'\\文件整理表格.xls'\n",
    "        df2 = pd.read_excel(fileday)\n",
    "        #df.set_index('日期',inplace = True)\n",
    "        df37=pd.concat([df37,df2])\n",
    "        df37.set_index('日期',inplace = True)\n",
    "        df37.to_excel(ppath1+'\\文件整理表格.xls')\n",
    "        print('数据已保存至：'+ ppath[:-11])\n",
    "        print('*'*50)\n",
    "\n",
    "     \n",
    "    def 未备份列表():         \n",
    "        for a in ['cak','wl','ha']:\n",
    "            path371 = os.path.join(r'\\\\10.153.110.37\\Data\\solar_observation\\2021',a)\n",
    "            网盘37列表=[]#文件列表\n",
    "            for root,dirs,files in os.walk(path371):\n",
    "                for i in dirs:\n",
    "                    #print(i)\n",
    "                    if len(i) == 11:  #针对10/20之前的文件命名\n",
    "                        网盘37列表.append(int(i[:8]))\n",
    "                    #elif len(i) >13 and i !='processed' and i !='AS_F1000':  #新的文件命名方式\n",
    "                        #网盘37列表.append(int(i[:8]))\n",
    "                    elif len(i) == 10 and i !='AS_F1000' and i !='processed':  #wlpco 2021.5.30添加\n",
    "                        网盘37列表.append(int(i[:8]))\n",
    "\n",
    "            #对文件夹进行排列\n",
    "            网盘37列表.sort(reverse=True)   \n",
    "            df = pd.read_excel(path371+'\\文件整理表格.xls')\n",
    "            set37 = set(df.iloc[:,0])\n",
    "            网盘37字典[a] = set(网盘37列表) - set37\n",
    "        return 网盘37字典\n",
    "\n",
    "    #表格抬头 Z是字典\n",
    "    #num =日期\n",
    "    def all(num):\n",
    "        #os.chdir(r'E:\\shuju\\num')\n",
    "        os.chdir(ppath)\n",
    "        mtime = [] #存放上午时间段\n",
    "        ntime = []#存放下午时间段\n",
    "        size = 0 #文件大小\n",
    "        lensize = 0#文件数量\n",
    "        for i in os.scandir(): #单个文件夹资料\n",
    "            if i.name.endswith('.fit') or i.name.endswith('.fits'):\n",
    "                #文件大小\n",
    "                size += os.path.getsize(i)/1024/1024\n",
    "                #文件数量\n",
    "                lensize +=1\n",
    "                #print(str('%.2f' %size)+'M')\n",
    "                #print(os.path.getctime(i))\n",
    "                #dt.datetime.fromtimestamp(os.path.getmtime(i).append(mtime)\n",
    "\n",
    "                #创建时间循环\n",
    "                mmtime=dt.datetime.fromtimestamp(os.path.getmtime(i))\n",
    "                if mmtime.hour < 13:   #小于13时的时间放入mmtime列表内\n",
    "                    mtime.append(mmtime)\n",
    "                elif 13 < mmtime.hour < 19: #大于13时小于19时的时间放入nmtime列表内\n",
    "                    ntime.append(mmtime)\n",
    "        #输出需要的资料，并打包\n",
    "        #print(os.path.split(os.getcwd())[1][:-3])\n",
    "        mtime.sort() #修复时间段乱序问题\n",
    "        ntime.sort() #修复时间段乱序问题\n",
    "        日期.append(文件夹日期(os.path.split(os.getcwd())[1]))\n",
    "        #print(时间段(mtime))\n",
    "        观测时间段1.append(时间段(mtime))\n",
    "        #print(时间段(ntime))\n",
    "        观测时间段2.append(时间段(ntime))\n",
    "        #print(时间计算(mtime)+时间计算(ntime))\n",
    "        总观测时间.append(时间计算(mtime)+时间计算(ntime))\n",
    "        #print(lensize)\n",
    "        观测文件.append(lensize)\n",
    "        #print('文件总大小'+str('%.2f' %size)+'M')\n",
    "        文件总大小.append(str('%.2f' %size)+'M')\n",
    "        黑子(re.search('\\d+',str(num)).group())\n",
    "\n",
    "    #在ppath填入CAK/WL/HA路径\n",
    "    #day = input('请输出需要整理的日期，比如：20211011CAK,整理文件将放在当日文件夹下')\n",
    "    #print('你选择的日期是'+day)\n",
    "\n",
    "    #未备份列表() 获得每个通道没备份的文件名 字典形式网盘37字典={}\n",
    "    未备份列表()\n",
    "    for 三通道 in 网盘37字典: #遍历‘cak''ha''wl'\n",
    "        print(f'正在处理{三通道}文件')\n",
    "        for path37 in 网盘37字典[三通道]: #日期\n",
    "            fname=[]\n",
    "            日期=[]\n",
    "            观测时间段1=[]\n",
    "            观测时间段2=[]\n",
    "            总观测时间=[]\n",
    "            观测文件=[]\n",
    "            文件总大小=[]\n",
    "            编号=[]\n",
    "            方位=[]\n",
    "            黑子数量=[]\n",
    "            耀斑爆发等级和时间=[]\n",
    "            z={'日期':日期,'观测时间段1':观测时间段1,'观测时间段2':观测时间段2,'总观测时间':总观测时间,'观测文件':观测文件,'文件总大小':文件总大小,'编号':编号,'方位':方位,'黑子数量':黑子数量,'耀斑爆发等级和时间':耀斑爆发等级和时间}\n",
    "            try:\n",
    "                if 三通道 == 'cak':\n",
    "                    ppath=os.path.join(r'\\\\10.153.110.37\\Data\\solar_observation',str(path37)[:4],'CAK',str(path37)+'CAK')\n",
    "                    ppath1=ppath[:-11]\n",
    "                elif 三通道 == 'wl':\n",
    "                    ppath=os.path.join(r'\\\\10.153.110.37\\Data\\solar_observation',str(path37)[:4],'WL',str(path37)+'WL')\n",
    "                    ppath1=ppath[:-10]\n",
    "                elif 三通道 =='ha':\n",
    "                    ppath=os.path.join(r'\\\\10.153.110.37\\Data\\solar_observation',str(path37)[:4],'HA',str(path37)+'HA')\n",
    "                    ppath1=ppath[:-10]\n",
    "                os.chdir(ppath)\n",
    "                print('目标地址：'+ppath)\n",
    "\n",
    "                #c处理的最终主程序\n",
    "                all(path37)\n",
    "                #处理detafram\n",
    "                df = pd.DataFrame.from_dict(z)\n",
    "                df.set_index('日期',inplace = True)\n",
    "                #print(df.head) #需要打印基本信息时删掉前面的#\n",
    "                #更改文件存放位置\n",
    "                os.chdir(ppath)\n",
    "                df.to_excel(ppath+'\\文件整理表格.xls')\n",
    "                df.drop(df.index,inplace=True)\n",
    "                print('已完成，文件放置日期处理目录：'+ppath)\n",
    "                合并() #合并数据\n",
    "            except FileNotFoundError:\n",
    "                print(path37+'目录下没有当天日期')\n",
    "                pass\n",
    "    print('数据分析完成，期待明天有好天气')\n",
    "        \n",
    "schedule.every().day.at(\"18:30\").do(job) #18：30为启动任务时间\n",
    "#schedule.every().day.at(\"15:07\").do(job) #10：50为启动任务时间\n",
    "while True:\n",
    "    schedule.run_pending()\n",
    "    #time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cak': set()}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "网盘37字典"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cak': {20211001, 20211002, 20211003, 20211004, 20211005},\n",
       " 'wl': {20211003, 20211004, 20211005},\n",
       " 'ha': {20211001, 20211002, 20211003, 20211004, 20211005}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "网盘37字典={}\n",
    "for a in ['cak','wl','ha']:\n",
    "    path371 = os.path.join(r'\\\\10.153.110.37\\Data\\solar_observation\\2021',a)\n",
    "    网盘37列表=[]#文件列表\n",
    "    for root,dirs,files in os.walk(path371):\n",
    "        for i in dirs:\n",
    "            #print(i)\n",
    "            if len(i) == 11:  #针对10/20之前的文件命名\n",
    "                网盘37列表.append(int(i[:8]))\n",
    "            #elif len(i) >13 and i !='processed' and i !='AS_F1000':  #新的文件命名方式\n",
    "                #网盘37列表.append(int(i[:8]))\n",
    "            elif len(i) == 10 and i !='AS_F1000' and i !='processed':  #wlpco 2021.5.30添加\n",
    "                网盘37列表.append(int(i[:8]))\n",
    "                \n",
    "    #对文件夹进行排列\n",
    "    网盘37列表.sort(reverse=True)   \n",
    "    df = pd.read_excel(path371+'\\文件整理表格.xls')\n",
    "    set37 = set(df.iloc[:,0])\n",
    "    网盘37字典[a] = set(网盘37列表) - set37\n",
    "    #return 网盘37字典\n",
    "网盘37字典"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'dict' object is not callable",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-14-355046c0e91e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     20\u001b[0m         \u001b[0m网盘37字典\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m网盘37列表\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mset37\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     21\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0m网盘37字典\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 22\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0m网盘37字典\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     23\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m网盘37字典\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: 'dict' object is not callable"
     ]
    }
   ],
   "source": [
    "def wmm():\n",
    "    网盘37字典={}\n",
    "    for a in ['cak','wl','ha']:\n",
    "        path371 = os.path.join(r'\\\\10.153.110.37\\Data\\solar_observation\\2021',a)\n",
    "        网盘37列表=[]#文件列表\n",
    "        for root,dirs,files in os.walk(path371):\n",
    "            for i in dirs:\n",
    "                #print(i)\n",
    "                if len(i) == 11:  #针对10/20之前的文件命名\n",
    "                    网盘37列表.append(int(i[:8]))\n",
    "                #elif len(i) >13 and i !='processed' and i !='AS_F1000':  #新的文件命名方式\n",
    "                    #网盘37列表.append(int(i[:8]))\n",
    "                elif len(i) == 10 and i !='AS_F1000' and i !='processed':  #wlpco 2021.5.30添加\n",
    "                    网盘37列表.append(int(i[:8]))\n",
    "\n",
    "        #对文件夹进行排列\n",
    "        网盘37列表.sort(reverse=True)   \n",
    "        df = pd.read_excel(path371+'\\文件整理表格.xls')\n",
    "        set37 = set(df.iloc[:,0])\n",
    "        网盘37字典[a] = set(网盘37列表) - set37\n",
    "    return 网盘37字典"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{20211001, 20211002, 20211003, 20211004, 20211005}\n",
      "{20211003, 20211004, 20211005}\n",
      "{20211001, 20211002, 20211003, 20211004, 20211005}\n"
     ]
    }
   ],
   "source": [
    "for i in 网盘37字典:\n",
    "    print(网盘37字典[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "path371 = os.path.join(r'\\\\10.153.110.37\\Data\\solar_observation\\2021','cak')\n",
    "网盘37列表=[]#文件列表\n",
    "for root,dirs,files in os.walk(path371):\n",
    "    for i in dirs:\n",
    "        if len(i) == 11:  #针对10/20之前的文件命名\n",
    "            网盘37列表.append(int(i[:8]))\n",
    "        elif len(i) >13 and i !='processed':  #新的文件命名方式\n",
    "            网盘37列表.append(int(i[:8]))\n",
    "        elif len(i) == 8:  #wlpco 2021.5.30添加\n",
    "            网盘37列表.append(int(i[:8]))\n",
    "#对文件夹进行排列\n",
    "网盘37列表.sort(reverse=True)   \n",
    "df = pd.read_excel(path371+'\\文件整理表格.xls')\n",
    "set37 = set(df.iloc[:,0])\n",
    "网盘37字典[i] = set(网盘37列表) - set37"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'cak' == 'CAK'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'20210910CAK': {20211001, 20211002, 20211003, 20211004, 20211005}}"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "网盘37字典"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
